package cc.yihy.spider.core;

import cc.yihy.spider.entity.Article;
import cc.yihy.spider.service.ArticleService;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;

/**
 * @Package: cc.yihy
 * @date: 2017/11/21 16:14
 * @author: Yihy
 * @version: V1.0
 * @Description:
 */
@Component
public class DBPipeline implements Pipeline {


    @Autowired
    private ArticleService articleService;


    public DBPipeline() {

    }

    @Override
    public void process(ResultItems resultItems, Task task) {

        if (StringUtils.isBlank((String)resultItems.get("title")) || StringUtils.isBlank((String)resultItems.get("content"))) {
            return;
        }

        String url = resultItems.getRequest().getUrl();
        String id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."));

        String content = resultItems.get("content");
        content = removeEmoji(content);
        String title = resultItems.get("title");
        title = removeEmoji(title);


        Article article = new Article();
        article.setContent(content);
        article.setTitle(title);
        article.setUrl(url);
        article.setId(Integer.valueOf(id));
        articleService.save(article);
    }

    private String removeEmoji(String title) {
        if (title != null) {
            title = title.replaceAll("[^\\u0000-\\uFFFF]", "");

        }
        return title;
    }


}
